package com.bigdata.spark.core.wc

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object Spark01_WordCount {
  def main(args: Array[String]): Unit = {
    //spark框架，建立和spark框架的连接
    //JDBC：connections
    val sparkConf = new SparkConf().setMaster("local").setAppName("WordCount");
    val sc = new SparkContext(sparkConf);
    //处理业务
    //1.读取文件，获取一行一行的数据
    val lines: RDD[String] = sc.textFile("datas");
    //2.将一行数据拆分成一个一个的单词
    val words: RDD[String] = lines.flatMap(_.split(" "));

    //3.将数据进行分组
    //RDD[(String, Iterable[String])]  这是一个Tuple，第一个是单词，第二是可迭代的集合
    val wordGroup: RDD[(String, Iterable[String])] = words.groupBy(word=>word)
    //4.对分组后的数据进行转换
    val wordCount = wordGroup.map {
      case (word, list) => {
        (word, list.size)
      }
    }
    //5.输出到控制台打印
    val array: Array[(String, Int)] = wordCount.collect()
    array.foreach(println)
    //关闭连接
    sc.stop();
  }
}
